package com.university.report.cmn.handler;

import lombok.extern.slf4j.Slf4j;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.ArrayList;
import java.util.List;

/**
 * 用于爬取高风险地区的 PageProcessor
 * @ClassName web-magic-test
 * @Author Jinondo
 * @Date 2021/6/20 20:09
 */
@Slf4j
public class ImpRegionPageProcessor implements PageProcessor {
    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site
            .me()
            .setTimeOut(10000)
            .setRetryTimes(3)
            .setSleepTime(1000)
            .setCharset("UTF-8")
            .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
    ;

    // 獲得站點
    @Override
    public Site getSite() {
        return site;
    }



    //爬取過程
    @Override
    public void process(Page page) {
        //获取当前页的所有风险地区
        List<String> heightList = page.getHtml().xpath("//div[@class='height']/div/ul/li").all();
        List<String> middleList = page.getHtml().xpath("//div[@class='middle']/div/ul/li").all();

        //要保存风险地区的集合
        List<String> heights = new ArrayList<>();
        List<String> middles = new ArrayList<>();
        //遍历风险地区

        for (String item : heightList) {
            Html tmp = Html.create(item);
            String region = tmp.xpath("//span/text()").toString();
            heights.add(region);

        }
        for (String item : middleList) {
            Html tmp = Html.create(item);
            String region = tmp.xpath("//span/text()").toString();
            middles.add(region);

        }
        log.info("高风险地区->" + heights.toString());
        log.info("中风险地区->" + middles.toString());

        List<List<String>> res = new ArrayList<List<String>>(){{
            add(heights);
            add(middles);
        }};
        page.putField("importantRegions", res);
    }
}
